########################################################################
#  Copyright (C) 2007,8,9 Ehud Ben-Reuven
#  udi@benreuven.com
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
########################################################################
r"""Flat files
All incoming statements are flat lines:
Statements have a relatively shallow and rigid structrue, in order to simplfy
the processing and make all formats of bank statements look the same, the
statements are transformed into lines that are completely flat:
each line contains all the information it needs without any reference to other
lines.

A line is made from pairs of tag and value.
All the pairs in the same line are assumed to be bound togther and describe
different aspects of the same object(s)
(e.g., bank, account, holding, transaction or asset).


The syntax of each line is:
<flat-line> = [SP <tag><op><value> ]*
<tag> = <a3Tag> | <ofxTag>
<ofxTag> = [A-Z]+ # tag from ofx: name space
<a3Tag> = [a-z][a-zA-Z0-9]* # tag from a3: name space
<op> = <relational-op>
<relational-op> = '='
<value> = <text-without-blanks> | ...
   (QUOTE <any-text> QUOTE) | ... # Surounding quotes on values are removed
   '`' <python-expression> '`' | #  Evaluate the text surronded by a back-strich
                                 # as a python expression and cast to string
    
* You can insert a quote by \"
* &quot; is translated back to a quote char.
* You can put a '#' comment at an end of a flat line
* empty lines are ignored
* The parse method converts lines to a dictionary in which the tag is the key
  and its value is a list of all the values the same tag received in the line.
  As a result the order in which pairs appeared in the line is not preserved.

None flat aspects of a file of flat lines:
* All lines are assumed to come from the file even if the file name itself does
  not appear in each line this is trivial but it has some
  implications. For example: two lines in the same file showing a transaction
  from the same holding on the same time will be assumed to be two different
  transactions if they come from the same file, however the same two lines
  coming from two different files are more likely to be two different
  discriptions of the same transaction. Note that the file basename itself will
  be used as a reference, without its path or extension or any other
  signature so different unique names should be used for different flat files.
"""

import os
import re
import sys
from rdflib.graph import *
from rdflib.namespace import *
from n3tagvalue import *
OFX = Namespace("http://www.w3.org/2000/10/swap/pim/ofx#")
A3 = Namespace("http://code.google.com/p/3account/wiki/Schema#")
XSD = Namespace("http://www.w3.org/2001/XMLSchema#")

FS = Namespace("file://")
flatLine = A3["flatLine"]

reobj=re.compile(r"""(?x) # allow for commenting inside the RE
\s* # Ignore white space
(
 (?P<comment>\#)| # Ignore the rest of the line.
 ( # or look for a regular tag-value pair
  (?P<tag>\w+) # tag
  = # look for operator
  ( # Value
   (?P<quote>"|`)? # look for quote at start of value
   (?P<value>.*?)  # finally the value. Dont be greedy so not to eat the
                    # next token. Note that an empty value is also OK
   (?(quote)
    ((?<!\\)(?P=quote))|  # If there was a quote at the start then look 
                           # for it at the end. Skip backslash quote
    (?=(\s|$))    # If there wasn't a quote at start: The value is not
                 # greedy so force an unquoted value to be followed with
                 # white-space or or to continue to end of string
   )  
  ) #end of value
 ) # end of tag-value pair
)""")

def parse(line):
    """Parse a line from a flat file into its tag-value pairs.
    The return-value is a dictionary where the keys are the tags and the
    values are the list of the values that appeared in the line.
    """
    if not line: return {}
    l=line
    ldict={}
    while True:
        p=reobj.match(l)
        if not p: break

        # handle the return value in line for a recursive usage
        l=l[p.end():]
        if p.group('comment'):
            return ldict

        tag=p.group('tag')
        values=ldict.setdefault(tag,[]) # Make sure the tag appears in ldict even if there is no value.
        quote=p.group('quote')
        value=p.group('value')
        value=value.replace('&quot;','"')
        if not value: # empty values don't need to be added to list of values for tag in ldict
            pass
        elif quote=='`':
            values.append(eval(value))
        else:
            values.append(value)
    return ldict

def flat(fin,fout,context=None,hashing=False):    
    if isinstance(fin,str):
        fin=open(fin, "r")
    else:
        fin.seek(0)
    
    n3header(fout,"$Id$"[1:-1],context)

    for line in fin:
        line = parse(line)
        if not line: continue
        print >>fout,"a3:flatLine [ ",
        for tag,value in line.iteritems():
            print >>fout,stagvalue(tag,value,hashing=hashing),
        print >>fout,"];"
    print >>fout,"."
    fin.close()

    